package Crawl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.concurrent.ArrayBlockingQueue;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class JsoupTest {
	
	
	public static void main(String[] args) {

		PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
		CloseableHttpClient httpClient = HttpClients.custom()
		        .setConnectionManager(cm)
		        .build();
		String	oriurl="http://cec.jmu.edu.cn/";
		String oriurl1="http://cec.jmu.edu.cn/list.jsp";
		
		ArrayList lis1=new ArrayList();	
		ArrayList lis2=new ArrayList();	
		ArrayList lis3=new ArrayList();	
		ArrayList lis4=new ArrayList();	
		
		
		
		try {
			Document doc=Jsoup.connect("http://cec.jmu.edu.cn/").get();
			Elements links = doc.select(".menu0_0_");  
			for (Element link : links) {  
                lis1.add(oriurl+link.attr("href"));
            }  
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		
		
		
		GetThread[] threads=new GetThread[lis1.size()];
		for(int i=0;i<threads.length;i++){
			 HttpGet httpget = new HttpGet(lis1.get(i).toString());
			 threads[i] = new GetThread(httpClient, httpget);
		}
		
		
		for (int j = 0; j < threads.length; j++) { //�ܾ�û��д���߳��ˣ����start����������
		    threads[j].start();
		}

		// ���һ����������
		for (int j = 0; j < threads.length; j++) {
		    try {
				threads[j].join();
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}	
		
		for (int i=0;i<threads.length;i++)
		{
			for(int j=0;j<threads[i].getLis1().size();j++){
				lis2.add(threads[i].getLis1().get(j));
			}
	
		}
		
		for (int i=0;i<threads.length;i++)
		{
			for(int j=0;j<threads[i].getLis().size();j++){
				lis3.add(oriurl1+threads[i].getLis().get(j));
			}
	
		}
		
		for ( int i = 0 ; i < lis3.size() - 1 ; i ++ ) {   //��������ѭ��  
		     for ( int j = lis3.size() - 1 ; j > i; j -- ) {  //����������ѭ��  
		       if (lis3.get(j).equals(lis3.get(i))) {  
		         lis3.remove(j);                            
		       }   
		      }   
		    }   
		
		GetThread1[] threads1=new GetThread1[lis3.size()];
		for(int i=0;i<threads1.length;i++){
			 HttpGet httpget = new HttpGet(lis3.get(i).toString());
			 threads1[i] = new GetThread1(httpClient, httpget);
		}
		
		
		for (int j = 0; j < threads1.length; j++) { //�ܾ�û��д���߳��ˣ����start����������
		    threads1[j].start();
		}

		// ���һ����������
		for (int j = 0; j < threads1.length; j++) {
		    try {
				threads1[j].join();
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}	
		
		
		
		
		
		for (int i=0;i<threads1.length;i++)
		{
			for(int j=0;j<threads1[i].getLis().size();j++){
				lis3.add(oriurl+threads1[i].getLis().get(j));
			}
		}
		
	
for(int i=0;i<lis2.size();i++){
	
lis4.add(lis2.get(i));
}

for(int i=0;i<lis3.size();i++){
	lis4.add(lis3.get(i));
}

HashSet h = new HashSet(lis4);
lis4.clear();
lis4.addAll(h);


//for(int i=0;i<lis4.size();i++){
//	System.out.println(lis4.get(i));
//}   全部url存放的地方
		
GetThread3[] threads3 = new GetThread3[lis4.size()];
for (int i = 0; i < threads3.length; i++) {
    HttpGet httpget = new HttpGet(lis4.get(i).toString());
    threads3[i] = new GetThread3(httpClient, httpget);
}

// start the threads
for (int j = 0; j < threads3.length; j++) {
    threads3[j].start();
}

// join the threads
for (int j = 0; j < threads3.length; j++) {
    try {
		threads3[j].join();
	} catch (InterruptedException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
}







		
	}
}

	
